
/** 
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 */
package com.tompai.datagather;

import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileReader;
import java.io.FileWriter;
import java.io.IOException;
import java.io.Reader;
import java.io.Writer;
import java.util.ArrayList;
import java.util.List;

import org.apache.commons.io.IOUtils;
import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import com.tompai.datagather.common.MybatisUtil;
import com.tompai.datagather.dao.TwitterAcountSeedsMapper;
import com.tompai.datagather.dao.TwitterAcountsMapper;
import com.tompai.datagather.model.TwitterAcountSeeds;
import com.tompai.datagather.model.TwitterAcounts;
import com.tompai.datagather.twitter.acount.TwitterFollowPage;
import com.tompai.datagather.twitter.acount.TwitterHomeInfo;

/**
 * @desc: datagather-tools
 * @name: TwitterFOFCrawler.java
 * @author: tompai
 * @createTime: 2018年5月8日 上午9:51:35
 * @history:
 * @version: v1.0
 */

public class TwitterAcountCrawlerDemo {
	public static final Logger LOG = LoggerFactory.getLogger(TwitterAcountCrawlerDemo.class);

	/**
	 * @author: tompai
	 * @createTime: 2018年5月8日 上午9:51:35  
	 * @history:
	 * @param args
	 */
	public static void main(String[] args) {
		// TODO Auto-generated method stub
		//read index from a files ;
		String filePath="./src/main/resources/index.txt";
		int index = 0;
		String seed = null;
		// String homeUrl;
		String roleA = "following";
		String roleB = "followers";
		//读取历史采集索引文件
		File indexFile=new File(filePath);
		List<String> indexs=new ArrayList<String>();
		try {
			Reader input = new FileReader(indexFile);
			indexs=IOUtils.readLines(input);
			//Predicate<String> predicate = s -> s.startsWith("#");
			//indexs.removeIf(s->s.startsWith("#"));
			//indexs.removeIf(predicate);
			index=Integer.parseInt(indexs.get(0));
			input.close();
			//indexFile.delete();
		} catch (FileNotFoundException e) {
			e.printStackTrace();
		} catch (IOException e) { 
			e.printStackTrace();
		}
		//开始处理数据库文件//
		//TwitterAcounts acounts = new TwitterAcounts();
		int accountNumber=0;
		TwitterAcountsMapper acountsMapper = MybatisUtil.getSession().getMapper(TwitterAcountsMapper.class);
		TwitterAcountSeeds seeds = new TwitterAcountSeeds();
		TwitterAcountSeedsMapper seedsMapper = MybatisUtil.getSession().getMapper(TwitterAcountSeedsMapper.class);
		TwitterHomeInfo twitterHomeInfo = new TwitterHomeInfo();
		TwitterFollowPage followPage = new TwitterFollowPage();
		// TwitterFollowSearch followSearch= new TwitterFollowPage(); //
		do {
			// get the seed account //
			index++;
			seeds = seedsMapper.selectByPrimaryKey(Long.valueOf(index));
			if (seeds != null) {
				String acount = seeds.getAcount();
				if (StringUtils.isEmpty(acount)) {
					continue;
				}
				LOG.info("======================================================");
				LOG.info("Index:" + index + " - " + " Current acount: " + acount);
				accountNumber = acountsMapper.selectScreenNameNumber(acount);
				if (accountNumber != 0) {
					LOG.info("Index:" + index + " - Have visited: " + acount);
				} else {
					seed = seeds.getAcount();
					if (StringUtils.isEmpty(seed)) {
						continue;
					}
					LOG.info("Index:" + index + " - " + " Begain twitter acount: " + seed);
					boolean nextStep=twitterHomeInfo.parseHomePage(seed);
					// =============//
					if(nextStep) {
						LOG.info("Index:" + index + " - Begain  twitter follow : " + seed);
						followPage.parseFollow(seed, roleA);
						//followPage.parseFollow(seed, roleB);
					}else {
						LOG.info("Index:" + index + " - Begain  twitter follow : " + seed+" information Failed.");
					}
					// get the records//
					int accounts = acountsMapper.selectAccountNumber();
					LOG.info("the total account numbers:" + accounts);
				}

			} else {
				LOG.info("The Seeds list is Empty,Please add a acount in the seeds list ,then begainning.");
				//break;
			}
			//文件存储更新索引编号，以防异常中断后遍历索引丢失//
			try {
				//OutputStream outputStream=FileOutputStream(path);  
				Writer output = null;
				if(!indexFile.exists()) {
					indexFile.createNewFile();
					output=new FileWriter(indexFile);
				}else {
					//不追加写入//
					output=new FileWriter(indexFile, false);
				}
				indexs.clear();
				indexs.add(String.valueOf(index));
				//IOUtils.write(String.valueOf(index), output);
				IOUtils.writeLines(indexs, IOUtils.LINE_SEPARATOR, output);  
				output.close();
				LOG.info("Have update the account index:"+index);
			} catch (FileNotFoundException e) {
				e.printStackTrace();
			} catch (IOException e) {
				e.printStackTrace();
			}
			//索引写入完毕//
		} while (true);
	}

}
